package Temp.Delete;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WebPageReader {

        List<String> parseWebPage(String url) throws MalformedURLException,
                        IOException {
                URL l_url = new URL(url);
                String pageContent = getPageContent(l_url);
                List<String> emails = getEmail(pageContent);
                for (String email : emails) {
                        System.out.println(email + ", " + url);
                }
                List<String> links = retrieveLinks(l_url, pageContent);
                return links;
        }

        private String getPageContent(URL l_url) {
                try {
                        String sCurrentLine = "";
                        HttpURLConnection l_connection = (HttpURLConnection) l_url
                                        .openConnection();
                        l_connection.connect();
                        InputStream l_urlStream = l_connection.getInputStream();
                        BufferedReader l_reader = new BufferedReader(new InputStreamReader(
                                        l_urlStream));
                        StringBuffer pageContent = new StringBuffer();
                        while ((sCurrentLine = l_reader.readLine()) != null) {
                                pageContent.append(sCurrentLine);
                        }
                        handleFrames(l_url, pageContent);
                        return removeCommmentFromPageContent(pageContent.toString());
                } catch (Exception e) {
                        return "";
                }
        }

        // remove comments from web page with regEx
        private String removeCommmentFromPageContent(String pageContent) {
                return pageContent.replaceAll("<!--((?!-->).)+-->", "");
        }

        // the frame should be retreated as well as the whole page
        private void handleFrames(URL url, StringBuffer content) throws IOException {
                Pattern p = Pattern
                                .compile("<frame\\s+((?!<frame).)+src=\"(\\w+\\.\\w+)\">");
                Matcher m = p.matcher(content);
                while (m.find()) {
                        String frameurl = m.group(2).trim();
                        String urlPath = url.getPath();
                        if (!urlPath.endsWith("/")) {
                                urlPath = urlPath + "/";
                        }
                        URL l_url = new URL("http://" + url.getHost() + ":" + url.getPort()
                                        + urlPath + frameurl);
                        String frameContent = getPageContent(l_url);
                        content.append(frameContent);
                }
        }

        // read email from the web page with regEx
        private List<String> getEmail(String pageContent) {
                Pattern p = Pattern.compile("\\w+-*\\w+@(\\w+\\.)+[a-z]{2,3}",
                                Pattern.CASE_INSENSITIVE);
                Matcher m = p.matcher(pageContent);

                List<String> emails = new ArrayList<String>();
                while (m.find()) {
                        String email = m.group().trim();
                        if (!emails.contains(email))
                                emails.add(email);
                }
                return emails;
        }

        // read links from the web page with regEx
        private List<String> retrieveLinks(URL pageUrl, String pageContents) {
                Pattern p = Pattern.compile("<a\\s+href\\s*=\\s*\"?(.*?)[\"|>]",
                                Pattern.CASE_INSENSITIVE);
                Matcher m = p.matcher(pageContents);

                List<String> linkList = new ArrayList<String>();
                while (m.find()) {
                        String link = m.group(1).trim();

                        if (link.length() < 1) {
                                continue;
                        }
                        if (link.charAt(0) == '#') {
                                continue;
                        }
                        if (link.indexOf("mailto:") != -1) {
                                continue;
                        }
                        if (link.toLowerCase().indexOf("javascript") != -1) {
                                continue;
                        }
                        if (link.indexOf("://") == -1) {
                                if (link.charAt(0) == '/') {// deal with absolute address
                                        link = "http://" + pageUrl.getHost() + ":"
                                                        + pageUrl.getPort() + link;
                                } else {
                                        String file = pageUrl.getFile();
                                        if (file.indexOf('/') == -1) {// deal with relative
                                                // address
                                                link = "http://" + pageUrl.getHost() + ":"
                                                                + pageUrl.getPort() + "/" + link;
                                        } else {
                                                String path = file.substring(0,
                                                                file.lastIndexOf('/') + 1);
                                                link = "http://" + pageUrl.getHost() + ":"
                                                                + pageUrl.getPort() + path + link;
                                        }
                                }
                        }
                        if (!isLinkToWebpage(link))
                                continue;

                        if (!linkList.contains(link))
                                linkList.add(link);
                }
                return (linkList);
        }

        // is the link really linked to a web page, not a downloadable resource
        private boolean isLinkToWebpage(String link) {
                // remove http://
                String newLink = link.substring(7);
                String[] words = newLink.split("/");
                if (words.length == 1)
                        return true;
                String theLastWord = words[words.length - 1];
                if (!theLastWord.contains("."))
                        return true;
                else if (theLastWord.contains(".html") || theLastWord.contains(".htm")
                                || theLastWord.contains(".shtml")
                                || theLastWord.contains(".asp") || theLastWord.contains(".jsp")
                                || theLastWord.contains(".aspx")
                                || theLastWord.contains(".jspx")
                                || theLastWord.contains(".php") || theLastWord.contains("?"))
                        return true;
                else
                        return false;

        }
}

